from bs4 import BeautifulSoup
import re
import urllib

class HtmlParser(object):
    def _get_new_urls(self, url, soup):
        urls = set()

        links = soup.find_all('a', href=re.compile(r"/view/\d+\.html"))
        for link in links:
            new_url = link['href']
            new_full_url = urllib.parse.urljoin(url, new_url)
            urls.add(new_full_url)
        return urls

    def _get_new_data(self, url, soup):
        res_data = {}
        res_data['url'] = url
#<h2 class="title-text"><span class="title-prefix">Python</span>Python简介及应用领域</h2>

        title_node = soup.find('h2', class_="title-text")
        res_data['title'] = title_node.get_text()
        print(title_node.get_text())
# <div class="para" label-module="para">自从20世纪90年代初Python语言诞生至今，它已被逐渐广泛应用于系统管理任务的处理和<a target=_blank href="/item/Web/150564" data-lemmaid="150564">Web</a>编程。</div>
        summary_node = soup.find("div class=\"para\" label-module=\"para\"")
        res_data['summary'] = summary_node.get_text()
        print(summary_node.get_text())

        return res_data

    def parse(self, url, html):
        if url is None or html is None:
            return
        soup = BeautifulSoup(html, 'html.parser', from_encoding='utf-8')
        new_urls = self._get_new_urls(url, soup)
        new_data = self._get_new_data(url, soup)

        return new_urls, new_data
